# -*- coding: utf-8 -*-
import scrapy
import re
from copy import deepcopy
from scrapy import Request
from scrapy import log
from scienet.items import ScienetItem
from lxml import etree


class SciSpider(scrapy.Spider):
    name = 'sci'
    allowed_domains = ['blog.sciencenet.cn']
    base_url = "http://blog.sciencenet.cn/blog.php?mod=member&alphabet={}"
    blog_base_url = "http://blog.sciencenet.cn/home.php?mod=space&uid={}"

    def start_requests(self):
        for character in [chr(i) for i in range(65, 91)]:
            url = self.base_url.format(character)
            start_request = Request(url=url, callback=self.parse)
            start_request.headers["Referer"] = "http://blog.sciencenet.cn/blog.php?mod=member"
            yield start_request

    def parse(self, response):
        bloggers_list = response.xpath("//div[@class='potbox']")
        if bloggers_list.extract():
            for blogger in bloggers_list:
                blogger_url = blogger.xpath(".//a/@href").extract_first()
                if blogger_url:
                    item = ScienetItem()
                    item["uid"] = re.match(r".*?uid=(\d+)", blogger_url, re.S).group(1)
                    item["name"] = blogger.xpath(".//a/text()").extract_first()
                    # print(item)
                    # print(response.url)
                    if item["uid"]:
                        blog_url = self.blog_base_url.format(item["uid"])
                        blog_request = Request(url=blog_url, callback=self.parse_blog_page, meta={"item": item})
                        blog_request.headers["Referer"] = response.url
                        yield blog_request
        else:
            log.msg("<%s>没有相关博主!" % str(response.url), _level=log.WARNING)

    def parse_blog_page(self, response):
        item = response.meta.get("item")
        # print("+"*50)
        # print(item)
        if response.xpath("//div[@id='nv']").extract():
            # 获取头衔、机构、研究领域信息
            blogger_info_list = response.xpath("//li[@class='ul_diy']").extract()
            if blogger_info_list:
                for blogger_info in blogger_info_list:
                    if "工作情况" in blogger_info:
                        html = etree.HTML("<ul>" + blogger_info + "</ul>")
                        info = html.xpath("//li/text()")
                        if info:
                            info_list = info[0].split("，")
                            item["institute"] = info_list[0]
                            if len(info_list) == 3:
                                item["title"] = info_list[2]
                            elif "副研究员" in info[0]:
                                item["title"] = "副研究员"
                            elif "副教授" in info[0]:
                                item["title"] = "副教授"
                            elif "教授" in info[0]:
                                item["title"] = "教授"
                            elif "研究员" in info[0]:
                                item["title"] = "助理研究员"
                            elif "助理研究员" in info[0]:
                                item["title"] = "研究员"
                            elif "讲师" in info[0]:
                                item["title"] = "讲师"
                            elif ("薄厚" in info[0] or "博后" in info[0] or "博士后" in info[0]):
                                item["title"] = "博士后"
                            elif "工程师" in info[0]:
                                item["title"] = "工程师"
                            else:
                                item["title"] = "其他"
                    elif "教育情况" in blogger_info:
                        html = etree.HTML("<ul>" + blogger_info + "</ul>")
                        info = html.xpath("//li/text()")
                        if info:
                            info_list = info[0].split("，")
                            item["institute"] = info_list[0]
                            if ("博士" in info[0] or "博士在读" in info[0]):
                                item["title"] = "博士"
                            elif ("硕士" in info[0] or "硕士在读" in info[0]):
                                item["title"] = "硕士"
                            elif ("本科" in info[0] or "本科在读" in info[0]):
                                item["title"] = "本科"
                            elif "教授" in info[0]:
                                item["title"] = "教授"
                            elif "副教授" in info[0]:
                                item["title"] = "副教授"
                            else:
                                item["title"] = "其他"
                    elif "研究领域" in blogger_info:
                        html = etree.HTML("<ul>" + blogger_info + "</ul>")
                        info = html.xpath("//li/text()")
                        if info:
                            info_list = info[0].split("->")
                            if info_list[0]:
                                item["realm"] = info_list[0]
                                if len(info_list) == 3:
                                    item["realm_medium"] = info_list[1]
                                    item["realm_type"] = info_list[2]
                                elif len(info_list) == 2:
                                    item["realm_medium"] = info_list[1]
            # 获取博客访问总数
            visitors = response.xpath("//div[@id='pcd']/p/strong/text()").extract_first()
            if visitors and (not visitors == "--"):
                item["visitors"] = int(visitors)
            else:
                item["visitors"] = 0

            # 获取博客博文总数
            articles = response.xpath("//div[@id='pcd']/ul/li[7]/a/text()").extract()
            if len(articles) == 2:
                articles.pop(articles.index("学术名片"))
                if articles[0] == "--":
                    item["articles"] = 0
                else:
                    item["articles"] = int(articles[0])
            else:
                item["articles"] = 0

            # 获取博客活跃度
            activity = response.xpath("//div[@id='pcd']/ul/li[4]/a/text()").extract()
            if len(activity) == 2:
                activity.pop(activity.index("给我留言"))
                if activity[0] == "--":
                    item["activity"] = 0
                else:
                    item["activity"] = int(activity[0])
            else:
                item["activity"] = 0

        elif re.findall(r"<li>博客访问量: (.*?)</li>", response.body.decode("gbk"), re.S):
            # 如果博主设置了访问权限，则从提示页提取访问量、博文数量、活跃度信息
            visitors = re.findall(r"<li>博客访问量: (.*?)</li>", response.body.decode("gbk"), re.S)[0].strip()
            articles = re.findall(r"<li>帖子数: (.*?)</li>", response.body.decode("gbk"), re.S)[0].strip()
            activity = re.findall(r"<li>活跃度: (.*?) ℃</li>", response.body.decode("gbk"), re.S)[0].strip()
            if visitors.isdigit():
                item["visitors"] = int(visitors)
            else:
                item["visitors"] = 0
            if articles.isdigit():
                item["articles"] = int(articles)
            else:
                item["articles"] = 0
            if activity.isdigit():
                item["activity"] = int(activity)
            else:
                item["activity"] = 0
        else:
            log.msg("<uid:%s>可能已被封博！" % item["uid"], _level=log.WARNING)
            # print("="*50)
            # print(item)

        # 如果获取到的头衔仍旧为空或者未能获取到
        if not item.get("title"):
            item["title"] = "其他"
        # 如果获取到的单位仍旧为空或者未能获取到
        if not item.get("institute"):
            item["institute"] = "其他"
        # 如果未能获取到一级学科信息
        if (not item.get("realm")) or (item.get("realm") == "0"):
            item["realm"] = "未知"
        # 如果未能获取到二级学科信息
        if not item.get("realm_medium"):
            item["realm_medium"] = "未知"
        # 如果未能获取到三级学科信息
        if not item.get("realm_type"):
            item["realm_type"] = "未知"
        yield item





